import requests
import numpy
import json
import pandas
import sys
import pytz
from lxml import etree
import time

ids, names = [], []
# 不适用科学计数
numpy.set_printoptions(suppress=True)
# 纳斯达克100指数
response = requests.get(
    "https://cn.investing.com/equities/StocksFilter?noconstruct=1&smlID=595&sid=&tabletype=price&index_id=20",
    headers={"User-Agent": "Mozilla/5.0"})
htmlList = etree.HTML(response.text).xpath('//table[@id="cross_rate_markets_stocks_1"]/tbody/tr')
for i in range(len(htmlList)):
    id = htmlList[i].get('id').split("_")[1]
    ids.append(id)
    columns = htmlList[i].xpath('./td')
    name = columns[1].xpath('./a/text()')[0]
    names.append(name)
    high = columns[3].xpath('./text()')
    low = columns[4].xpath('./text()')
    涨跌幅 = columns[6].xpath('./text()')
    交易量 = columns[7].xpath('./text()')

# 标准普尔500指数
response = requests.get(
    "https://cn.investing.com/equities/StocksFilter?noconstruct=1&smlID=595&sid=&tabletype=price&index_id=166",
    headers={"User-Agent": "Mozilla/5.0"})
htmlList = etree.HTML(response.text).xpath('//table[@id="cross_rate_markets_stocks_1"]/tbody/tr')
for i in range(len(htmlList)):
    id = htmlList[i].get('id').split("_")[1]
    # 去重
    if ids.count(id) == 0:
        ids.append(id)
        columns = htmlList[i].xpath('./td')
        name = columns[1].xpath('./a/text()')[0]
        names.append(name)

# 获取500+股每5分钟的数据
# ids = [251, 252]
dict = {}
for i in range(len(ids)):
    timestamp, dateList, hourList, minuteList, secondList = [], [], [], [], []
    openPriceList, closePriceList, highPriceList, lowPriceList, volumeList, voList = [], [], [], [], [], []
    # 纽约时间 2021-02-03 00:00:00 = 时间戳 1612328400
    # 纽约时间 2021-11-10 00:00:00 = 时间戳 1636520400
    # 请求有效间隔 86400  是一天的秒数
    # 美股交易时间: 美国东部时间9:30-16:00 夏令: 8:30-15:00
    # 美国时间比中国时间晚13小时  两者时间戳是一样的
    s, e = 0, 1612328399
    while e < 1636520400:
        time.sleep(1)
        s = e + 1
        # 每次获取两天172800
        e = s + 3000000
        response = requests.get(
            "https://tvc4.investing.com/f95326d4f4e43603b4366f86d37c4bf1/1636379784/6/6/28/history?symbol=" +
            str(ids[i]) + "&resolution=5&from=" + str(s) + "&to=" + str(e), headers={"User-Agent": "Mozilla/5.0"})
        data = json.loads(response.text)
        # 跳过没有数据的节假日
        if data['s'] == 'no_data':
            continue
        timestamp.extend(data['t'])
        openPriceList.extend(data['o'])
        closePriceList.extend(data['c'])
        highPriceList.extend(data['h'])
        lowPriceList.extend(data['h'])
        volumeList.extend(data['v'])
        voList.extend(data['vo'])
        # 时间戳取并集
        for k in range(len(data['t'])):
            datetime = pytz.datetime.datetime.fromtimestamp(data['t'][k], pytz.timezone('America/New_York'))
            # dateList.append(datetime.strftime('%Y-%m-%d %H:%M:%S'))
            hourList.append(datetime.strftime('%H'))
            minuteList.append(datetime.strftime('%M'))
    channel = []
    # if len(timestamp) != len(dict['时间']):
    #     continue
    for j in range(len(timestamp)):
        # 剔除重复数据 - -!
        if len(numpy.where(numpy.asarray(timestamp[0:j]) == timestamp[j])[0]) > 0:
            continue
        # 通道: [开盘价, 收盘价, 最高价, 最低价, 成交量, 成交量2, 小时, 分钟]
        channel.append([openPriceList[j], closePriceList[j], highPriceList[j], lowPriceList[j], volumeList[j], voList[j], int(hourList[j]), int(minuteList[j]), timestamp[j]])
    dict[names[i]] = channel
    if i == 0:
        dict['时间'] = timestamp
    else:
        dict['时间'] = sorted(set(timestamp).union(set(dict['时间'])))

# 缺省补全 缺省的数据用0填充(不剔除 也不用平均填充),  直接在缺省的数据上进行训练, 尽可能保留最多信息
for j in range(len(ids)):
    oldChannel = dict[names[j]]
    for i in range(len(dict['时间'])):
        # 下面方法太慢
        # try:
        #     # numpy.where(numpy.asarray(oldChannel, dtype='l')[:, -1]==1612362600)
        #     list(numpy.asarray(oldChannel, dtype='l')[:, -1]).index(dict['时间'][i])
        # except BaseException:
        #     oldChannel.insert(i, [0,0,0,0,0,0,0,0,0])
        for k in range(i, len(oldChannel)):
            if dict['时间'][i] == oldChannel[k][-1]:
                break
            if oldChannel[k][-1] > dict['时间'][i]:
                oldChannel.insert(i, [0, 0, 0, 0, 0, 0, 0, 0, 0])
                break

# 清理长度大于时间的字典, 不懂为什么长度会大于, 明明时间是其他所有字典的并集了
for key in list(dict):
    if len(dict[key]) != len(dict['时间']):
        del dict[key]

# 时间格式化
times = dict['时间']
datetimes = []
for t in range(len(times)):
    datetime = pytz.datetime.datetime.fromtimestamp(times[t], pytz.timezone('America/New_York'))
    datetimes.append(datetime.strftime('%Y-%m-%d %H:%M'))
dict['时间'] = datetimes

# 写文件
dataFrame = pandas.DataFrame(dict)
# 时间放第一列
other = list(dict.keys())
other.remove("时间")
new = ['时间']
new.extend(other)
dataFrame = dataFrame[new]
dataFrame.to_csv("../temp/data/英为财情/标准普尔500.csv", index=False, sep=';', encoding='gbk')

sys.exit(1)
